//#define MY_WINDOWS // TODO : change it after constructing make file

#ifdef MY_WINDOWS
#include "DataCollector.h"
#include "DataTypeDetector.h"
#include "Dictionary.h"
#include "TextParser.h"
#include "WikiParser.h"
#include "Indexer.h"
#include "Parameters.h"
#include "TimeAnalysis.h"
#include "fwIndexer.h"
#else
#include "CommonSettingsAndUtilities.cpp"
#include "DataCollector.cpp"
#include "DataTypeDetector.cpp"
#include "Dictionary.cpp"
#include "TextParser.cpp"
#include "WikiParser.cpp"
#include "Indexer.cpp"
#include "parameters.cpp"
#include "TimeAnalysis.cpp"
#include "fwIndexer.cpp"
#endif
using namespace std;

int main(int argc, char* argv[])
{
	// input
	time_t start = time(NULL);

	#ifdef MY_WINDOWS
	parameters p(argv[1]);
	#else
	parameters p(argv[2]);
	#endif
	string newsDirectoryPath(p.path_input_dir + "/news");
	string wikiDirectoryPath(p.path_input_dir + "/wiki");

	// output
	string outputDirectory(p.path_dict_dir);

	// create output directory
	CreateNewDirectory(outputDirectory);
	CreateNewDirectory(outputDirectory + "/SemWiki");
	CreateNewDirectory(outputDirectory + "/Dictionary");
	CreateNewDirectory("./index_temp_files");
	CreateNewDirectory(p.path_index_dir);

	// scan for data
	DataCollector dataCollector(outputDirectory, 10000);
	dataCollector.ScanForNewDocuments(newsDirectoryPath);
	dataCollector.ScanForNewDocuments(wikiDirectoryPath);

	// pre-process and parse data
	Document			doc("", 0, "");
	Document			semWiki("", 0, "");
	DataTypeDetector	dataTypeDetector;
	DATA_TYPE			dataType = T_DEFAULT;

	TextParser textParser(outputDirectory);
	WikiParser wikiParser(outputDirectory);
	bool dumpMetadata	= true;
	index_class indexObject(p);
	fwIndexer fwi(p);
	
	while(dataCollector.GetNextDocument(doc))
	{
		// pre-process data 
		dataType	= dataTypeDetector.GetDataType(doc.m_sPath);

		switch(dataType)
		{
			case T_NEWS:
				// do nothing
				break;
			case T_WIKI:
				// remove metadata
				wikiParser.PreProcessDocument(doc, semWiki, dumpMetadata);
				// add new sem wiki doc to file dic
				semWiki.m_iDocID	= dataCollector.AddNewDocumentToDict(semWiki.m_sPath);
				break;
			case T_DEFAULT:
				// throw error
				cout << "Error: Data type detection failed." << endl;
				break;
		}

		// parse data
		textParser.ParseDocument(doc.m_sData);		

		indexObject.update_index(textParser.tokens, doc.m_iDocID);

		if(dataType == T_WIKI)
			fwi.makeIndex(semWiki.m_iDocID, semWiki.m_sData, textParser.TermDict.m_mDict);

	
		cout << doc.m_iDocID << endl;
		
	}

	indexObject.write_to_file();  //write whatever is left in index (less than 50000 postings)
	indexObject.MergeIndex();
	fwi.writeIndex(); //write whatever is in forward index

	// dump dictionaries
	cout << "dumping dictionaries\n";
	dataCollector.DumpFileDictionary();
	wikiParser.DumpDictionaries();
	textParser.DumpDicts();

	time_t end = time(NULL);
	cout << "time taken: " << end - start <<"s" <<endl;
	return 0;
}
